package com.gary.spider;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class Gather implements Runnable {

	private Dispatcher disp;
	private String ID;
	private File file;
	private BufferedWriter bfWriter;

	public Gather(String ID) {
		this.disp = Dispatcher.getDispatcher();
		String   fs   =   System.getProperties().getProperty("file.separator");
		file = new File("Raws"+fs+"RAW__" + ID + ".txt");
		try {
			file.createNewFile();
			bfWriter = new BufferedWriter(new FileWriter(file));
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	public String getDocAndUrl() {
		String url = disp.getURL();
		Document doc = null;
		try {
			doc = Jsoup.connect(url).get();
		} catch (IOException e) {
			e.printStackTrace();
		}
		Elements links = doc.select("a[href]");
		//doc.title(url);
		Iterator iter = links.iterator();
		ArrayList urllist = new ArrayList();
		while (iter.hasNext()) {
			Element u = (Element) iter.next();
			String a_href = u.toString();
			String[] geturl1 = a_href.split("href=\"");
			String[] geturl2 = geturl1[1].split("\\?|\">");
			String geturl = geturl2[0];
			Pattern pattern = Pattern.compile("^(http|https|ftp|etc)");
			Matcher matcher = pattern.matcher(geturl.subSequence(0,
					geturl.length() - 1));
			if (matcher.find()) {
				urllist.add(geturl);
			}
		}
		disp.insert(urllist);
		return doc.toString();
	}

	public void run() {
		try {
			bfWriter.write(getDocAndUrl());
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
}
